import datetime
import operator
import json
from distutils.util import strtobool
from collections import OrderedDict, defaultdict
from itertools import chain
from django.db.models import Prefetch, Q
from django.http import Http404
from django.utils.dateparse import parse_date
from django.utils.decorators import method_decorator
from django.views.decorators.cache import cache_page
from django_filters.rest_framework import DjangoFilterBackend
from rest_framework.generics import get_object_or_404
from rest_framework.mixins import RetrieveModelMixin

from ztp_backend.permissions import IsUUIDPermission
from rest_framework import views, viewsets, generics, mixins, status, filters
from rest_framework.decorators import api_view
from rest_framework.exceptions import ValidationError
from rest_framework.filters import SearchFilter, OrderingFilter
from rest_framework.response import Response

from mongolog.models import CASLog, StrategyLog, FileLog, ShareLog, AppsLog, LoopholesRiskLog, ProgramsRiskLog, \
    AttackDetectionLog, BasicSafetyLog
from mongolog.utils import date_range
from tools import range_date
from tools.datetime import date_filter, yesterday, as_datetime, to_date, date_filter_new
from tools.pagination import PageSizePagination
from asset.models import DeviceInfo, ResourceSensLevel, Device
from zmad.models import ZTPResource, ZTPUser, ZTPDevice, ZTPGroup
from userdata.models import UserAddScore
from strategy.models import Strategy
from ztp_backend.settings import APPS_RISK_TYPE_MAP, DEFAULT_PLATFORM
from .filters import TerminalFilter, FileFilter
from .serializers import (
    asset_count_serializer, trend_graph_serializer,
    malware_dist_serializer,
    TerminalSerializer, TerminalRetrieveSerializer,
    FileSerializer, most_visits, PCTerminalSerializer,
    TerminalRiskSerializer, TerminalDeviceSerializer
)
from userdata.serializers import device_geotrack_serializer


class AppVisitCountView(views.APIView):
    """应用同比/环比访问量"""

    # @method_decorator(cache_page(60 * 10))
    def get(self, request, *args, **kwargs):
        abnormal = request.query_params.get('abnormal')
        platform = request.query_params.get('platform')
        if not platform:
            platform = DEFAULT_PLATFORM
        else:
            if platform not in list(map(str, range(1, 5))):
                res = Response({"msg": "platform参数错误", "data": {}},
                               status=status.HTTP_500_INTERNAL_SERVER_ERROR)
                return res
            platform = int(platform)
        if abnormal:
            abnormal_count = asset_count_serializer(CASLog, platform=platform, filter_resource=True,
                                                    auth_code=[6, 8])
            data = abnormal_count
        else:
            data = asset_count_serializer(CASLog, platform=platform, filter_resource=True,
                                          auth_code=[5, 6, 7, 8])
        return Response(data)


# class ResourceListViewSet(mixins.RetrieveModelMixin,
#                      viewsets.GenericViewSet):
#     """企业应用列表选择项"""
#     queryset = ZTPResource.objects.all()
#
#     def list(self, request, *args, **kwargs):
#         """应用列表"""
#         queryset = self.get_queryset().values_list('pk', 'name').order_by('pk')
#         data = [{'id': pk, 'name': name} for pk, name in queryset]
#         return Response(data)

# @method_decorator(cache_page(60 * 10), 'list')
class AppListViewSet(mixins.RetrieveModelMixin,
                     viewsets.GenericViewSet):
    """企业应用访问量列表&详情"""
    queryset = ZTPResource.objects.all()
    pagination_class = PageSizePagination

    def list(self, request, *args, **kwargs):
        """全部应用访问量列表"""
        queryset = self.get_queryset().values_list('pk', 'name', "sen_mark").order_by('pk')
        search = request.query_params.get('search')
        if search:
            queryset = queryset.filter(name__iexact=search)
        platform = request.query_params.get('platform')
        if not platform:
            platform = DEFAULT_PLATFORM
        else:
            if platform not in list(map(str, range(1, 5))):
                res = Response({"msg": "platform参数错误", "data": {}},
                               status=status.HTTP_500_INTERNAL_SERVER_ERROR)
                return res
            platform = int(platform)

        data = OrderedDict(
            (pk, {'id': pk, 'name': name, 'visit_count': 0, 'user_count': 0,
                  'abnormal_count': 0, "sen_mark": sen_mark})
            for pk, name, sen_mark in queryset
        )
        # 访问量，访问用户数
        cas_aggregation = [
            {'$match': {'auth_code': {'$in': [5, 6, 7, 8]}}},
            {
                '$group': {
                    '_id': '$resource',
                    'visit_count': {'$sum': 1},
                    'username': {'$addToSet': '$username'}
                }
            },
            {
                '$project': {
                    'visit_count': 1,
                    'user_count': {'$size': '$username'}
                }
            }
        ]
        if platform:
            cas_aggregation[0]['$match']['platform'] = platform
        queryset1 = CASLog.objects.mongo_aggregate(cas_aggregation)
        for agg in queryset1:
            try:
                res_id = agg.pop('_id')
                data[res_id].update(agg)
            except KeyError:
                pass

        # 异常访问量
        abnormal_cas_aggregation = [
            {
                '$match': {
                    'auth_code': {'$in': [6, 8]}
                }
            },
            {
                '$group': {
                    '_id': '$resource',
                    'abnormal_count': {'$sum': 1}
                }
            }
        ]
        if platform:
            abnormal_cas_aggregation[0]['$match']['platform'] = platform
        queryset2 = CASLog.objects.mongo_aggregate(abnormal_cas_aggregation)
        for agg in queryset2:
            try:
                res_id = agg.pop('_id')
                data[res_id].update(agg)
            except KeyError:
                pass

        data = list(data.values())
        abnormal = request.query_params.get('abnormal')
        if abnormal:
            abnormal = bool(strtobool(abnormal))
            op = operator.gt if abnormal else operator.eq
            data = list(filter(lambda x: op(x['abnormal_count'], 0), data))

        op = operator.gt
        data = list(filter(lambda x: op(x['visit_count'], 0), data))
        # 按照访问量进行排序
        data = sorted(data, key=lambda item: item['visit_count'], reverse=True)
        page = self.paginate_queryset(data)
        if page is not None:
            return self.get_paginated_response(page)
        return Response(data)

    def retrieve(self, request, *args, **kwargs):
        """单个应用访问量"""
        res = self.get_object()
        platform = request.query_params.get('platform')
        if not platform:
            platform = DEFAULT_PLATFORM
        else:
            if platform not in list(map(str, range(1, 5))):
                res = Response({"msg": "platform参数错误", "data": {}},
                               status=status.HTTP_500_INTERNAL_SERVER_ERROR)
                return res
            platform = int(platform)
        date_from = as_datetime(datetime.date.today())
        date_to = as_datetime(datetime.date.today() + datetime.timedelta(days=1))
        data = {'name': res.name, 'visit_count': 0, 'user_count': 0, 'abnormal_count': 0,
                "create_time": res.crated_time, "icon": res.icon, "icon_base64": res.icon_base64, "today_user_count": 0,
                "today_visit_count": 0}
        # 访问量，访问用户数
        cas_aggregation1 = [
            {'$match': {'resource': res.id,
                        'auth_code': {'$in': [5, 6, 7, 8]}}},
            {
                '$group': {
                    '_id': None,
                    'visit_count': {'$sum': 1},
                    'username': {'$addToSet': '$username'}
                }
            },
            {
                '$project': {
                    '_id': 0,
                    'visit_count': 1,
                    'user_count': {'$size': '$username'}
                }
            }
        ]
        if platform:
            cas_aggregation1[0]['$match']['platform'] = platform
        queryset = CASLog.objects.mongo_aggregate(cas_aggregation1)
        try:
            data.update(next(queryset))
        except StopIteration:
            pass
        # 今日访问量, 今日访问用户数
        cas_aggregation2 = [
            {'$match': {'resource': res.id,
                        'log_time': {"$gte": date_from, "$lte": date_to},
                        'auth_code': {'$in': [5, 6, 7, 8]}}},
            {
                '$group': {
                    '_id': None,
                    'today_visit_count': {'$sum': 1},
                    'username': {'$addToSet': '$username'}
                }
            },
            {
                '$project': {
                    '_id': 0,
                    'today_visit_count': 1,
                    'today_user_count': {'$size': '$username'}
                }
            }
        ]
        if platform:
            cas_aggregation2[0]['$match']['platform'] = platform
        queryset1 = CASLog.objects.mongo_aggregate(cas_aggregation2)
        try:
            data.update(next(queryset1))
        except StopIteration:
            pass
        # 异常访问量
        # todo 这里统计不对,应该用Cas log来过滤,但是目前接口好像没用到,暂不改
        if platform:
            data['abnormal_count'] = StrategyLog.objects.filter(
                resource=res.id,
                uid__startswith='STG/RES',
                platform=platform
            ).count()
        else:
            data['abnormal_count'] = StrategyLog.objects.filter(
                resource=res.id,
                uid__startswith='STG/RES'
            ).count()
        return Response(data)


class AppVisitHistoryV2(generics.GenericAPIView):
    queryset = ZTPResource.objects.all()
    pagination_class = PageSizePagination

    def options(self, request, *args, **kwargs):
        res = Strategy.objects.filter(category__in=['资源请求']).values("name").distinct().values_list("name")
        data = [i[0] for i in res]
        return Response(data)

    def get(self, request, *args, **kwargs):
        res = self.get_object()
        try:
            date_from = datetime.datetime.strptime(request.query_params.get("dateFrom"), "%Y-%m-%d")
            date_to = datetime.datetime.strptime(request.query_params.get("dateTo"), "%Y-%m-%d")
            date_from = as_datetime(date_from)
            date_to = as_datetime(date_to)
        except:
            date_from, date_to = None, None
        if date_to:
            date_to = date_to + datetime.timedelta(days=1)
        platform = request.query_params.get('platform')
        # platform = '3'
        if not platform:
            platform = DEFAULT_PLATFORM
        else:
            if platform not in list(map(str, range(1, 5))):
                res = Response({"msg": "platform参数错误", "data": {}},
                               status=status.HTTP_500_INTERNAL_SERVER_ERROR)
                return res
            platform = int(platform)
        # platform = 3
        # 获取渠道对应描述对应关系字段
        channel_str_tuple = Strategy.STRATEGY_CHANNELS
        username2name = dict(ZTPUser.objects.values_list('username', 'name'))
        username2department_id = dict(ZTPUser.objects.values_list('username', 'belong_to_id'))
        uuid2name = dict(Device.objects.values_list('device_uuid', 'name'))
        stguid2name = dict(Strategy.objects.values_list('uid', 'name'))
        uid2channel = dict((Strategy.objects.values_list('uid', 'channel')))  # 和渠道关联
        if uid2channel:
            for key_value in uid2channel.keys():
                uid2channel[key_value] = channel_str_tuple[uid2channel[key_value]][1]

        # 获取策略name，uid对应关系字段
        stg_uid_dict = dict()
        for s in Strategy.objects.all():
            if not stg_uid_dict.get(s.name):
                stg_uid_dict[s.name] = [s.uid]
            else:
                stg_uid_dict[s.name].append(s.uid)
        platform2channel = dict(Strategy.STRATEGY_CHANNELS)
        platform2channel[None], platform2channel[0] = "-", "-"
        # platform2channel = {1: "Android", 2: "Windows", 3: "Web", 4: "Linux", 0: "-", None: "-"}
        if platform:
            if not date_from and not date_to:
                cas_queryset = CASLog.objects.filter(resource=res.id, auth_code__in=[5, 6, 7, 8], platform=platform).order_by('-log_time')
            else:
                date_to = date_to or as_datetime(datetime.datetime.now())
                date_from = date_from or as_datetime(datetime.datetime.strptime("2000-01-01", "%Y-%m-%d"))
                cas_queryset = CASLog.objects.filter(resource=res.id, auth_code__in=[5, 6, 7, 8],
                                                 platform=platform, log_time__gte=date_from, log_time__lte=date_to).order_by('-log_time')
                
        else:
            if not date_from and not date_to:
                cas_queryset = CASLog.objects.filter(resource=res.id, auth_code__in=[5, 6, 7, 8]).order_by('-log_time')
            else:
                date_to = date_to or as_datetime(datetime.datetime.now())
                date_from = date_from or as_datetime(datetime.datetime.strptime("2000-01-01", "%Y-%m-%d"))
                cas_queryset = CASLog.objects.filter(resource=res.id, auth_code__in=[5, 6, 7, 8], log_time__gte=date_from, log_time__lte=date_to).order_by('-log_time')
        
        strategy = request.query_params.get('strategy', '')
        # 将strategy转变成对应的uid
        if stg_uid_dict.get(strategy):
            # strategy过滤
            cas_queryset = cas_queryset.filter(strategy__in=stg_uid_dict.get(strategy))
            # 策略表过滤
            # strategy_queryset = strategy_queryset.exclude(uid__isnull=True).exclude(uid='')
            # strategy_queryset = strategy_queryset.filter(uid__in=stg_uid_dict.get(strategy))
        # else:
        #     strategy_queryset = None
        search = request.query_params.get('search')
        cas_queryset = cas_queryset.exclude(username__isnull=True).exclude(username='')
        if search:  # 搜索用户名/姓名
            extra = [username for username, name in username2name.items() if search in name]
            if cas_queryset is not None:
                if len(extra) > 0:
                    cas_queryset = cas_queryset.filter(Q(username__icontains=search) | Q(username__in=extra))
                else:
                    cas_queryset = cas_queryset.filter(username__icontains=search)
            # if strategy_queryset is not None:
            #     strategy_queryset = strategy_queryset.filter(Q(username__icontains=search) | Q(username__in=extra))

        # if cas_queryset is not None:
        #     data = list(cas_queryset)
        #     data.extend(list(cas_queryset))
        # if strategy_queryset is not None:
        #     # data.extend(list(strategy_queryset))
        #     data += list(strategy_queryset)
        # data.sort(key=lambda x: x.log_time, reverse=True)
        # data = sorted(chain(strategy_queryset, cas_queryset), key=lambda x: x.log_time, reverse=True)

        # 只取cas
        data = cas_queryset.order_by("-log_time")
        page = self.paginate_queryset(data)

        if page is not None:
            page = [
                {
                    'username': i.username,
                    'name': username2name.get(i.username),
                    'department': ZTPGroup.objects.filter(pk=username2department_id.get(i.username)).first().name,
                    'strategy': i.meta.name or "安全" if isinstance(i, StrategyLog) else stguid2name.get(i.strategy, '安全'),
                    'log_time': i.auth_time.split('.')[0],  # 时间只保留到 秒。
                    "url": i.url,
                    "platform": platform2channel[i.platform] if isinstance(i, CASLog) else uid2channel[i.uid],
                    "device": uuid2name.get(i.device_uuid, "-"),
                    "location": i.ip_location.city_name or "-" if isinstance(i, CASLog) else "-",
                    "auth_result": i.auth_result or "-" if isinstance(i, CASLog) else "-"
                }
                for i in page
            ]
            return self.get_paginated_response(page)
        return Response(data)


class AppVisitHistory(generics.GenericAPIView):
    """
    应用访问历史记录列表
    只返回前100条
    """
    queryset = ZTPResource.objects.all()

    def get(self, request, *args, **kwargs):
        res = self.get_object()
        # 正常访问记录从CAS日志中取
        cas_queryset = CASLog.objects.filter(resource=res.id).order_by('-log_time')
        # 异常访问记录从策略日志中取
        strategy_queryset = StrategyLog.objects.filter(resource=res.id, uid__startswith='STG/RES') \
            .order_by('-log_time')
        strategy = request.query_params.get('strategy', '')
        if strategy == 'NORMAL':
            strategy_queryset = None
        elif strategy.startswith('STG/RES'):
            cas_queryset = None
            strategy_queryset = strategy_queryset.filter(uid=strategy)

        username2name = dict(ZTPUser.objects.values_list('username', 'name'))
        search = request.query_params.get('search')
        if search:  # 搜索用户名/姓名
            extra = [username for username, name in username2name.items() if search in name]
            if cas_queryset is not None:
                cas_queryset = cas_queryset.filter(Q(username__icontains=search) | Q(username__in=extra))
            if strategy_queryset is not None:
                strategy_queryset = strategy_queryset.filter(Q(username__icontains=search) | Q(username__in=extra))

        data = []
        if cas_queryset is not None:
            data.extend(list(cas_queryset[:100]))
        if strategy_queryset is not None:
            data.extend(list(strategy_queryset[:100]))
        data.sort(key=lambda x: x.log_time, reverse=True)
        data = [
            {
                'username': i.username,
                'name': username2name.get(i.username),
                'strategy': i.meta.name if isinstance(i, StrategyLog) else '正常访问',
                'log_time': i.log_time.strftime('%Y-%m-%d %H:%M:%S')
            }
            for i in data[:100]
        ]
        return Response(data)


# @cache_page(60 * 10)
@api_view(['GET'])
def app_visit_trend(request):
    """应用访问详情趋势图"""
    platform = request.query_params.get('platform')
    if not platform:
        platform = DEFAULT_PLATFORM
    else:
        if platform not in list(map(str, range(1, 5))):
            res = Response({"msg": "platform参数错误", "data": {}},
                           status=status.HTTP_500_INTERNAL_SERVER_ERROR)
            return res
        platform = int(platform)
    date_from, date_to = date_filter(request.query_params)
    # 应用名称
    resource = request.query_params.get("resource", None)
    # resource = int(request.query_params.get("resource", 0))
    if resource is None:
        # 获取默认7天内访问次数最多的应用
        resource = most_visits(date_from, date_to)
    if resource is not None:
        data = trend_graph_serializer(
            CASLog, 'resource', date_from, date_to, auth_code=[5, 6, 7, 8], resource=int(resource), platform=platform,
            **date_range('log_time', date_from, date_to)
        )
    else:
        data = {"resource": "", "trend": []}
    return Response(data)


@cache_page(60 * 10)
@api_view(['GET'])
def email_attachment_count(request):
    """邮件附件总数"""
    data = asset_count_serializer(FileLog, source=FileLog.SOURCE_EMAIL)
    # 涉密邮件附件
    abnormal = FileLog.objects.filter(
        source=FileLog.SOURCE_EMAIL,
        sen_level__gt=0
    ).count()
    data['abnormal'] = abnormal
    return Response(data)


@cache_page(60 * 10)
@api_view(['GET'])
def strategy_triggering_trend(request):
    """策略触发情况趋势图"""
    date_from, date_to = date_filter(request.query_params)
    data = trend_graph_serializer(
        StrategyLog, 'meta.category',
        **date_range('log_time', date_from, date_to)
    )
    return Response(data)


@cache_page(60 * 10)
@api_view(['GET'])
def url_visit_count(request):
    """内部URL访问量"""
    data = asset_count_serializer(CASLog)
    return Response(data)


@cache_page(60 * 10)
@api_view(['GET'])
def file_download_count(request):
    """内部文件下载量"""
    data = asset_count_serializer(FileLog, source=FileLog.SOURCE_DOWNLOAD)
    # 涉密文件数
    abnormal = FileLog.objects.filter(
        source=FileLog.SOURCE_DOWNLOAD,
        sen_level__gt=0
    ).count()
    data['abnormal'] = abnormal
    return Response(data)


@api_view(["POST"])
def sen_mark(request):
    res = dict()
    res["status"] = 0
    res["msg"] = "success"
    param = json.loads(request.body)
    sen_level = param.get("sen_mark", 0)
    resource_id = param.get("id")
    resource = ZTPResource.objects.filter(id=resource_id, enabled=True).all()
    if not resource_id or not resource:
        res["status"] = 1
        res["msg"] = "无效应用，请检查传入参数"
    rsl = ResourceSensLevel.objects.get_or_create(resource=resource[0])[0]
    rsl.sen_level = sen_level
    rsl.save()
    return Response(res)


@api_view(["GET"])
def safety_consciousness_event(request):
    res = dict()
    res["status"] = 0
    res["message"] = "success"
    res["data"] = list()
    uas_objects = UserAddScore.objects.order_by("-datetime")[0:10]
    for uas in uas_objects:
        temp_dict = dict()
        temp_dict["user_name"] = uas.user.username
        temp_dict["uninstall_app_score"] = uas.uninstall_app
        temp_dict["scan_code"] = uas.scan_code
        temp_dict["total"] = uas.total
        temp_dict["datetime"] = uas.datetime
        temp_dict["fingerprint_score"] = uas.fingerprint_login
        res["data"].append(temp_dict)
    return Response(res)


@api_view(["GET"])
def danger_trend(request):
    res = dict()
    res["status"] = 0
    res["msg"] = "success"
    res["data"] = dict()
    date_to = as_datetime(datetime.datetime.now())
    date_from = as_datetime(date_to - datetime.timedelta(days=60))
    queryset = AppsLog.objects.mongo_aggregate([
        {
            '$match': {
                'device_info.device_uuid': {
                    '$exists': True
                },
                'log_time': {"$gt": date_from, "$lt": date_to}
            },
        },
        {
            '$project': {
                'device_info': 1,
                'log_time': 1,
                'danger_app': {'$size': '$apps_danger'},
                'risk_app': {'$size': '$apps_risk'},
                'sensitive_app': {'$size': '$apps_sensitive'}
            }
        },
        {
            '$group': {
                '_id': {
                    'device_uuid': '$device_info.device_uuid',
                    'log_time': '$log_time'
                },
                'danger_num': {'$sum': "$danger_app"},
                'risk_num': {'$sum': '$risk_app'},
                "sensitive_app": {"$sum": "$sensitive_app"},
                'date': {'$push': "$log_time"}
            }
        }
    ])
    for query in queryset:
        date = to_date(query["date"][0])
        if not res["data"].get(date):
            res["data"][date] = dict()
            res["data"][date]["danger_num"] = 0
            res["data"][date]["risk_num"] = 0
            res["data"][date]["sensitive_app"] = 0
        res["data"][date]["danger_num"] += query["danger_num"]
        res["data"][date]["risk_num"] += query["risk_num"]
        res["data"][date]["sensitive_app"] += query["sensitive_app"]
    return Response(res)


@api_view(['GET'])
def file_share_count(request):
    """内部文件分享量"""
    data = asset_count_serializer(FileLog, source=FileLog.SOURCE_SHARE)
    # 涉密文件数
    abnormal = ShareLog.objects.filter(
        sen_level__gt=0
    ).count()
    data['abnormal'] = abnormal
    return Response(data)


@cache_page(60 * 10)
@api_view(['GET'])
def url_share_count(request):
    """内部URL分享量"""
    data = asset_count_serializer(ShareLog, type='url')
    return Response(data)


class DataCirculateTrendView(views.APIView):
    """数据下载/分享/发送趋势"""

    def get_count(self, model, **filters):
        return model.objects.filter(**filters).count()

    def get_trend(self, model, **filters):
        aggregation = [
            {
                '$group': {
                    '_id': '$date',
                    'count': {'$sum': 1}
                }
            }
        ]
        if filters:
            aggregation.insert(0, {'$match': filters})
        queryset = model.objects.mongo_aggregate(aggregation)
        return {parse_date(agg['_id']): agg['count'] for agg in queryset if agg.get('_id')}

    @method_decorator(cache_page(60 * 10))
    def get(self, request, *args, **kwargs):
        date_from, date_to = date_filter(request.query_params)
        date_query = date_range('log_time', date_from, date_to)
        # 文件下载
        file_dl_count = self.get_count(FileLog, source=FileLog.SOURCE_DOWNLOAD)
        file_dl_trend = self.get_trend(FileLog, source=FileLog.SOURCE_DOWNLOAD, **date_query)
        # 文件分享
        file_shr_count = self.get_count(FileLog, source=FileLog.SOURCE_SHARE)
        file_shr_trend = self.get_trend(FileLog, source=FileLog.SOURCE_SHARE, **date_query)
        # URL分享
        url_shr_count = self.get_count(ShareLog, type='url')
        url_shr_trend = self.get_trend(ShareLog, type='url', **date_query)
        # 邮件附件
        mail_attachment_count = self.get_count(FileLog, source=FileLog.SOURCE_EMAIL)
        mail_attachment_trend = self.get_trend(FileLog, source=FileLog.SOURCE_EMAIL, **date_query)
        # 找出最大和最小日期
        date_set = {*file_dl_trend.keys(),
                    *file_shr_trend.keys(),
                    *url_shr_trend.keys(),
                    *mail_attachment_trend.keys()}
        if date_set:
            date_from, date_to = min(date_set), max(date_set)
        else:
            date_from = date_to = datetime.date.today()
        # 组装数据
        fields = ('文件下载', '文件分享', 'URL分享', '邮件附件')
        top = [{'field': k, 'count': v} for k, v in
               zip(fields, (file_dl_count, file_shr_count, url_shr_count, mail_attachment_count))]
        trend = {date: {'date': date.strftime('%Y-%m-%d'), **{i: 0 for i in fields}}
                 for date in range_date(date_from, date_to)}
        for date, count in file_dl_trend.items():
            trend[date]['文件下载'] = count
        for date, count in file_shr_trend.items():
            trend[date]['文件分享'] = count
        for date, count in url_shr_trend.items():
            trend[date]['URL分享'] = count
        for date, count in mail_attachment_trend.items():
            trend[date]['邮件附件'] = count
        data = {'top': top, 'trend': list(trend.values())}
        return Response(data)


class TerminalViewSet(viewsets.ReadOnlyModelViewSet):
    """终端列表、终端详情"""
    # TODO 按IMEI分组查询每一组中日期最新的记录
    '''
    方式1：EXISTS子句
    SELECT * FROM ztp.asset_deviceinfo main
    WHERE EXISTS(
        SELECT * FROM(
            SELECT imei, MAX(date) latest
            FROM ztp.asset_deviceinfo
            GROUP BY imei
        ) t
        WHERE main.imei = t.imei AND main.date = t.latest
    );

    方式2：自连接
    SELECT * FROM ztp.asset_deviceinfo main
        INNER JOIN (SELECT imei, max(date) latest FROM ztp.asset_deviceinfo GROUP BY imei) t
        ON main.imei = t.imei and main.date = t.latest;
    '''
    queryset = DeviceInfo.objects.select_related('user')
    serializer_class = TerminalSerializer
    filter_backends = (SearchFilter, DjangoFilterBackend, OrderingFilter)
    filterset_class = TerminalFilter
    search_fields = ('user__username', 'user__phone', 'user__name', 'device_uuid', 'model', 'os_version')
    ordering_fields = ('app', 'danger_app', 'risk_app', 'sensitive_app')
    pagination_class = PageSizePagination
    lookup_field = 'device_uuid'

    def get_serializer_class(self):
        if self.action == 'retrieve':
            return TerminalRetrieveSerializer
        return super().get_serializer_class()


class TerminalDeviceView(mixins.ListModelMixin,
                         mixins.UpdateModelMixin,
                         viewsets.GenericViewSet,
                         RetrieveModelMixin):
    """终端列表、终端详情"""
    queryset = Device.objects.all()
    serializer_class = TerminalDeviceSerializer
    filter_backends = (DjangoFilterBackend,)
    filterset_fields = ('device_uuid',)
    lookup_field = 'device_uuid'

    """
    Retrieve a model instance.
    """
    def retrieve(self, request, *args, **kwargs):
        pk = kwargs.get('device_uuid')
        condition = get_object_or_404(Device, device_uuid=pk)  # replaced pk by id here
        result = TerminalDeviceSerializer(condition)  # used condition in serializer
        # print(result)
        return Response(result.data)


@api_view(['GET'])
def get_device_info(request):
    device_uuid = request.query_params.get("device_uuid")
    try:
        data = dict()
        data_info = Device.objects.filter(device_uuid=device_uuid).first()
        # field_names = ['device_uuid', 'model', 'os_version', 'mac_addr', 'username', 'name', 'group_name',
        # 'phone', 'email']
        data['device_uuid'] = device_uuid
        data['model'] = data_info.model
        data['os_version'] = data_info.os_version
        data['mac_addr'] = data_info.mac_addr
        data['user'] = dict()
        data['user']['username'] = data_info.user.username
        data['user']['group_name'] = data_info.user.belong_to.name
        data['user']['phone'] = data_info.user.phone
        data['user']['email'] = data_info.user.email
        data['user']['name'] = data_info.user.name
        return Response(data)
    except ValidationError:
        raise Http404()


class TerminalTrackView(views.APIView):
    """终端地理轨迹"""

    @method_decorator(cache_page(60 * 10))
    def get(self, request, *args, **kwargs):
        data = device_geotrack_serializer(show_devices=False)
        return Response(data)


class TerminalMalwareView(generics.GenericAPIView):
    """终端恶意应用分类统计"""

    def get(self, request, *args, **kwargs):
        maltype = kwargs['maltype']
        try:
            data = malware_dist_serializer(maltype)
            return Response(data)
        except ValidationError:
            raise Http404()


class FileView(mixins.ListModelMixin,
               generics.GenericAPIView):
    """文件传输列表"""
    queryset = FileLog.objects.order_by('-log_time')
    serializer_class = FileSerializer
    filter_backends = (DjangoFilterBackend, SearchFilter)
    filterset_class = FileFilter
    search_fields = ('filename', 'username', "sen_level", "sen_type", "sen_words")
    pagination_class = PageSizePagination

    def get_queryset(self):
        queryset = super().get_queryset()
        date_from, date_to = date_filter(self.request.query_params)
        date_from = as_datetime(date_from)
        date_to = as_datetime(date_to)
        if date_from:
            queryset = queryset.filter(log_time__gte=date_from)
        if date_to:
            queryset = queryset.filter(log_time__lte=date_to + datetime.timedelta(days=1))
        return queryset

    def get(self, request, *args, **kwargs):
        return self.list(request, *args, **kwargs)


class PCTerminalViewSet(mixins.ListModelMixin,
                        mixins.UpdateModelMixin,
                        viewsets.GenericViewSet):
    """终端列表、PC终端详情"""
    queryset = ZTPDevice.objects.all()
    serializer_class = PCTerminalSerializer
    pagination_class = PageSizePagination
    filter_backends = (DjangoFilterBackend,)
    filterset_fields = ('is_secure', 'device_type', 'platform')

    def get_queryset(self):
        queryset = ZTPDevice.objects.all()
        return queryset

    def list(self, request, *args, **kwargs):
        # 通过cas表，strategy表，构建 "device_uuid" 关联 strategy.name 表。
        queryset = self.filter_queryset(self.get_queryset())
        page = self.paginate_queryset(queryset)
        if page is not None:
            # 如果有分页，序列化分页后的结果
            serializer = self.get_serializer(page, many=True)
            user_id_list = [int(data_i["user_id"]) for data_i in serializer.data]
            user_queryset = ZTPUser.objects.filter(id__in=user_id_list).all()
            for data_i in serializer.data:
                ZTPUser_i = user_queryset.filter(id=int(data_i["user_id"])).first()
                data_i.update({"username": ZTPUser_i.username})
                data_i.update({"name": ZTPUser_i.name})
                data_i.update({"phone": ZTPUser_i.phone})

            return self.get_paginated_response(serializer.data)

        # 如果没有分页，序列化整个查询集
        serializer = self.get_serializer(self.filter_queryset(self.get_queryset()), many=True)
        user_id_list = [int(data_i["user_id"]) for data_i in serializer.data]
        user_queryset = ZTPUser.objects.filter(id__in=user_id_list).all()
        for data_i in serializer.data:
            ZTPUser_i = user_queryset.filter(id=int(data_i["user_id"])).first()
            data_i.update({"username": ZTPUser_i.username})
            data_i.update({"name": ZTPUser_i.name})
            data_i.update({"phone": ZTPUser_i.phone})

        return Response(serializer.data, status=status.HTTP_200_OK)


class TerminalRiskViewSet(mixins.ListModelMixin,
                          mixins.UpdateModelMixin,
                          viewsets.GenericViewSet):
    """终端列表、终端风险详情"""
    queryset = Device.objects.all()
    serializer_class = TerminalRiskSerializer
    pagination_class = PageSizePagination
    filter_backends = (DjangoFilterBackend, filters.SearchFilter,)
    filterset_fields = ('is_secure', 'device_type', 'platform')
    search_fields = ('user__username', 'user__phone', 'user__name', 'device_uuid', 'model', 'os_version')

    def get_queryset(self):
        # 修正 risk_list 和 is_secure 字段不匹配，同时兼容设备uuid在检测过程中出现变更的情况
        stguid2name = dict(Strategy.objects.values_list('uid', 'name'))
        risk_uid = ['STG/TERM-10', 'STG/TERM-11', 'STG/TERM-12', 'STG/ENV-16']  # 终端威胁的对应策略uid
        device_info = Device.objects.all()
        for device_ in device_info:
            device_uid = StrategyLog.objects.filter(
                device_uuid=device_.device_uuid,
                uid__in=risk_uid
            ).values('uid').distinct().values_list('uid')
            if device_uid:
                device_risk_list_0 = device_.risk_list  # device表里面设备记录的触发的策略
                device_risk_list_1 = [stguid2name.get(i[0]) for i in list(device_uid)]  # strategy 表记录的触发的策略
                device_.risk_list = list(set(device_risk_list_0 + device_risk_list_1))  # 更新设备表的终端风险策略
            # 同步更新终端的安全记录
            if device_.risk_list:
                device_.is_secure = False
            else:
                device_.is_secure = True

            device_.save()

        queryset = Device.objects.all()
        return queryset

    def list(self, request, *args, **kwargs):
        queryset = self.filter_queryset(self.get_queryset())
        page = self.paginate_queryset(queryset)
        if page is not None:
            # 如果有分页，序列化分页后的结果
            serializer = self.get_serializer(page, many=True)
            for data_i in serializer.data:
                data_i.update({"user_id": data_i['user']['id']})
                data_i.update({"username": data_i['user']['username']})
                data_i.update({"name": data_i['user']['name']})
                data_i.update({"phone": data_i['user']['phone']})
                if not data_i['risk_list']:
                    data_i.update({"is_secure": True})


            return self.get_paginated_response(serializer.data)

        # 如果没有分页，序列化整个查询集
        serializer = self.get_serializer(self.filter_queryset(self.get_queryset()), many=True)
        for data_i in serializer.data:
            data_i.update({"user_id": data_i['user']['id']})
            data_i.update({"username": data_i['user']['username']})
            data_i.update({"name": data_i['user']['name']})
            data_i.update({"phone": data_i['user']['phone']})
            if not data_i['risk_list']:
                data_i.update({"is_secure": True})

        return Response(serializer.data, status=status.HTTP_200_OK)


@api_view(['GET'])
def asset_trend(request):
    """资产趋势图"""
    date_from, date_to = date_filter(request.query_params)

    data = {'fields': 1, 'trend': list()}
    return Response(data)


@api_view(['GET'])
def loophole_risk(request):
    """漏洞风险"""
    data = dict()  # 返回的数据
    device_uuid = request.query_params.get('device_uuid', None)
    if device_uuid is not None:
        platform = Device.objects.filter(device_uuid=device_uuid).first().platform
        if platform in (2, 4):
            linux_query_mongo = LoopholesRiskLog.objects.mongo_aggregate([
                {
                    '$match': {
                        'device_uuid': device_uuid,
                    }
                },
                {
                    '$project': {
                        'date': 1,
                        'device_uuid': 1,
                        'serious': 1,
                        'high': 1,
                        'medium': 1,
                        'low': 1,
                        "log_time": 1
                    }
                },
                {
                    "$sort": {"device_uuid": 1, "log_time": -1}
                },
            ])

            # 组装数据
            def get_dict_data(query_i, risk_type):
                # level_dict = {1: "低危漏洞", 2: "中危漏洞", 3: "高危漏洞", 4: "严重漏洞"}
                level_dict = {"low": "低危漏洞", "medium": "中危漏洞", "high": "高危漏洞", "serious": "严重漏洞"}
                data_list = list()
                if query_i[risk_type]:
                    for i in range(len(query_i[risk_type])):
                        data_dict = dict()
                        data_dict["code"] = dict(query_i[risk_type][i])["code"]
                        data_dict["name"] = dict(query_i[risk_type][i])["name"]
                        data_dict["level"] = level_dict[risk_type]
                        # data_dict["level"] = level_dict[dict(query_i[risk_type][0])["level"]]
                        data_dict["date"] = query_i['log_time'].strftime('%Y-%m-%d %H:%M:%S%z')
                        data_dict["details"] = dict(query_i[risk_type][i])
                        data_dict["details"]["date"] = query_i['log_time'].strftime('%Y-%m-%d %H:%M:%S%z')
                        data_dict["details"]["level"] = data_dict["level"]
                        data_list.append(data_dict)

                return data_list

            total_list = list()
            # 组装数据
            data = {'serious': list(), 'high': list(), 'medium': list(), 'low': list()}
            for linux_query_i in linux_query_mongo:
                for key, value in data.items():
                    l_ = get_dict_data(linux_query_i, key)
                    data[key].extend(l_)
                    total_list.extend(l_)

                break

            key_list = list(data.keys())
            for key in key_list:
                if not data[key]:  # 空数组则不展示
                    del data[key]
            data['total'] = total_list

    return Response(data)


@api_view(['GET'])
def loophole_risk_count(request):
    """漏洞风险"""
    data = dict()  # 返回的数据
    device_uuid = request.query_params.get('device_uuid', None)
    if device_uuid is not None:
        platform = Device.objects.filter(device_uuid=device_uuid).first().platform
        if platform in (2, 4):
            linux_query_mongo = LoopholesRiskLog.objects.mongo_aggregate([
                {
                    '$match': {
                        'device_uuid': device_uuid,
                    }
                },
                {
                    '$project': {
                        'date': 1,
                        'device_uuid': 1,
                        'serious': 1,
                        'high': 1,
                        'medium': 1,
                        'low': 1,
                        "log_time": 1
                    }
                },
                {
                    "$sort": {"device_uuid": 1, "log_time": -1}
                },
            ])

            # 组装数据
            data = {'serious': 0, 'high': 0, 'medium': 0, 'low': 0}
            total = 0
            for linux_query_i in linux_query_mongo:
                for key, value in data.items():
                    if linux_query_i[key]:
                        data[key] += len(linux_query_i[key])
                        total += len(linux_query_i[key])
                break
            data['total'] = total

    return Response(data)


@api_view(['GET'])
def programs_risk(request):
    """恶意代码风险"""
    data = {}  # 返回的数据
    device_uuid = request.query_params.get('device_uuid', None)
    if device_uuid is not None:
        platform = Device.objects.filter(device_uuid=device_uuid).first().platform
        if platform in (2, 4):
            linux_query_mongo = ProgramsRiskLog.objects.mongo_aggregate([
                {
                    '$match': {
                        'device_uuid': device_uuid,
                    }
                },
                {
                    '$project': {
                        'device_uuid': 1,
                        'date': 1,
                        'trojans': 1,
                        'infection': 1,
                        'worm': 1,
                        'hack': 1,
                        'grey': 1,
                        'risk': 1,
                        'rubbish': 1,
                        'others': 1,
                        "log_time": 1
                    }
                },
                {
                    "$sort": {"device_uuid": 1, "log_time": -1}
                },
                {'$limit': 1}
            ])

            # 组装数据
            def get_dict_data(query_i, risk_type):
                data_list = list()
                if query_i[risk_type]:
                    for i in range(len(query_i[risk_type])):
                        data_dict = dict()
                        data_dict["hash"] = dict(query_i[risk_type][i])["hash"]
                        data_dict["name"] = dict(query_i[risk_type][i])["name"]
                        data_dict["category"] = dict(query_i[risk_type][i])["category"]
                        data_dict["path"] = dict(query_i[risk_type][i])["path"]
                        data_dict["date"] = query_i['log_time'].strftime('%Y-%m-%d %H:%M:%S%z')
                        data_list.append(data_dict)

                return data_list

            total_list = list()
            # 组装数据
            data = {'trojans': list(), 'infection': list(), 'worm': list(), 'hack': list(), 'grey': list(),
                    'risk': list(), 'rubbish': list(), 'others': list()}
            for linux_query_i in linux_query_mongo:
                for key, value in data.items():
                    l_ = get_dict_data(linux_query_i, key)
                    data[key].extend(l_)
                    total_list.extend(l_)
            key_list = list(data.keys())
            for key in key_list:
                if not data[key]:  # 空数组则不展示
                    del data[key]
            data['total'] = total_list
            # print("data : ", data.keys())

    return Response(data)


@api_view(['GET'])
def programs_risk_count(request):
    """恶意代码风险"""
    data = {}  # 返回的数据
    device_uuid = request.query_params.get('device_uuid', None)
    if device_uuid is not None:
        platform = Device.objects.filter(device_uuid=device_uuid).first().platform
        if platform in (2, 4):
            linux_query_mongo = ProgramsRiskLog.objects.mongo_aggregate([
                {
                    '$match': {
                        'device_uuid': device_uuid,
                    }
                },
                {
                    '$project': {
                        'device_uuid': 1,
                        'date': 1,
                        'trojans': 1,
                        'infection': 1,
                        'worm': 1,
                        'hack': 1,
                        'grey': 1,
                        'risk': 1,
                        'rubbish': 1,
                        'others': 1,
                        "log_time": 1
                    }
                },
                {
                    "$sort": {"device_uuid": 1, "log_time": -1}
                },
                {'$limit': 1}
            ])

            # 组装数据
            data = {'trojans': 0, 'infection': 0, 'worm': 0, 'hack': 0, 'grey': 0, 'risk': 0, 'rubbish': 0, 'others': 0}
            for linux_query_i in linux_query_mongo:
                for key, value in data.items():
                    data[key] += len(linux_query_i[key])

            data['total'] = sum(list(data.values()))
            # print("data : ", data)

    return Response(data)


@api_view(['GET'])
def attack_detection(request):
    """入侵检测"""
    data = {"results": list()}  # 返回的数据
    device_uuid = request.query_params.get('device_uuid', None)
    if device_uuid is not None:
        platform = Device.objects.filter(device_uuid=device_uuid).first().platform
        if platform in (2, 4):
            linux_query_mongo = AttackDetectionLog.objects.mongo_aggregate([
                {
                    '$match': {
                        'device_uuid': device_uuid,
                    }
                },
                {
                    '$project': {
                        'date': 1,
                        'device_uuid': 1,
                        'attack': 1,
                        "log_time": 1
                    }
                },
                {
                    "$sort": {"device_uuid": 1, "log_time": -1}
                },
                {'$limit': 1}
            ])
            # 危险等级转换为字符串描述（1低危,2中危,3高危）
            level_dict = {1: '低危', 2: '中危', 3: '高危'}
            data_list = list()
            for linux_query_i in linux_query_mongo:
                data_dict = dict()
                if linux_query_i['attack']:
                    for i in range(len(linux_query_i['attack'])):
                        data_dict["attack_type"] = dict(linux_query_i['attack'][i])["attack_type"]
                        data_dict["attack_name"] = dict(linux_query_i['attack'][i])["attack_name"]
                        data_dict["level"] = level_dict[dict(linux_query_i['attack'][i])["level"]]
                        data_dict["date"] = linux_query_i['log_time'].strftime('%Y-%m-%d %H:%M:%S%z')
                        data_dict["details"] = dict(linux_query_i['attack'][i])
                        data_dict["details"]["date"] = data_dict["date"]
                        data_dict["details"]["level"] = data_dict["level"]
                        data_list.append(data_dict)

            data["results"] = data_list

    return Response(data)


@api_view(['GET'])
def basic_safety(request):
    """安全基线"""
    data = {"results": list()}  # 返回的数据
    device_uuid = request.query_params.get('device_uuid', None)
    if device_uuid is not None:
        platform = Device.objects.filter(device_uuid=device_uuid).first().platform
        if platform in (2, 4):
            linux_query_mongo = BasicSafetyLog.objects.mongo_aggregate([
                {
                    '$match': {
                        'device_uuid': device_uuid,
                    }
                },
                {
                    '$project': {
                        'date': 1,
                        'device_uuid': 1,
                        'detection': 1,
                        "log_time": 1
                    }
                },
                {
                    "$sort": {"device_uuid": 1, "log_time": -1}
                },
            ])

            data_list = list()
            for linux_query_i in linux_query_mongo:
                if linux_query_i['detection']:
                    for i in range(len(linux_query_i['detection'])):
                        data_dict = dict()
                        data_dict["name"] = dict(linux_query_i['detection'][i])["name"]
                        data_dict["result"] = dict(linux_query_i['detection'][i])["result"]
                        data_dict["date"] = (linux_query_i['log_time']).strftime('%Y-%m-%d %H:%M:%S%z')
                        data_dict["details"] = dict(linux_query_i['detection'][i])
                        data_dict["details"]["date"] = data_dict["date"]
                        data_list.append(data_dict)
                    break

            data["results"] = data_list

    return Response(data)


@api_view(['GET'])
def get_apps_risk(request):
    """移动端风险详情"""
    data = dict()  # 返回的数据
    device_uuid = request.query_params.get('device_uuid', None)
    if device_uuid is not None:
        platform = Device.objects.filter(device_uuid=device_uuid).first().platform
        if platform == 1:
            app_query_mongo = AppsLog.objects.mongo_aggregate([
                {
                    '$match': {
                        'device_uuid': device_uuid,
                    }
                },
                {
                    '$project': {
                        'date': 1,
                        'device_uuid': 1,
                        'apps_risk': 1,
                        'apps_danger': 1,
                        "log_time": 1
                    }
                },
                {
                    "$sort": {"device_uuid": 1, "log_time": -1}
                },
            ])

            # 组装数据
            def get_dict_data(query_i, app_type):
                keys_list = ['md5', 'package_name', 'program_name', 'apk_path', 'virus_name', 'virus_description',
                             'is_system_apps'
                             ]
                data_list = []
                if query_i[app_type]:
                    for i in range(len(query_i[app_type])):
                        data_dict = dict()
                        # 对缺少某些字段数据进行补充处理
                        for keys_ in keys_list:
                            if keys_ not in list(query_i[app_type][i].keys()):
                                query_i[app_type][i][keys_] = None
                            data_dict[keys_] = dict(query_i[app_type][i])[keys_]
                        data_dict["date"] = query_i['log_time'].strftime('%Y-%m-%d %H:%M:%S%z')
                        # 提取病毒名前缀，进行分类
                        virus_type = ''
                        if data_dict["virus_name"]:
                            virus_type = data_dict["virus_name"].split('/')[0]
                        if virus_type in list(APPS_RISK_TYPE_MAP.keys()):
                            data_dict["risk_type"] = APPS_RISK_TYPE_MAP[virus_type]
                        else:
                            data_dict["risk_type"] = '风险软件'

                        data_list.append(data_dict)

                return data_list

            apps_risk_list, apps_danger_list, total_list = list(), list(), list()
            for app_query_i in app_query_mongo:
                if 'apps_risk' in list(app_query_i.keys()) and app_query_i['apps_risk']:
                    l_1 = get_dict_data(app_query_i, 'apps_risk')
                    apps_risk_list.extend(l_1)
                    total_list.extend(l_1)
                if 'apps_danger' in list(app_query_i.keys()) and app_query_i['apps_danger']:
                    l_2 = get_dict_data(app_query_i, 'apps_danger')
                    apps_danger_list.extend(l_2)
                    total_list.extend(l_2)

            if apps_risk_list:
                data['risk'] = apps_risk_list
            if apps_danger_list:
                data['danger'] = apps_danger_list
            data['total'] = total_list

    return Response(data)


@api_view(['GET'])
def get_apps_risk_count(request):
    """移动端风险详情"""
    data = dict()  # 返回的数据
    device_uuid = request.query_params.get('device_uuid', None)
    if device_uuid is not None:
        platform = Device.objects.filter(device_uuid=device_uuid).first().platform
        if platform == 1:
            app_query_mongo = AppsLog.objects.mongo_aggregate([
                {
                    '$match': {
                        'device_uuid': device_uuid,
                    }
                },
                {
                    '$project': {
                        'date': 1,
                        'device_uuid': 1,
                        'apps_risk': 1,
                        'apps_danger': 1,
                        "log_time": 1
                    }
                },
                {
                    "$sort": {"device_uuid": 1, "log_time": -1}
                },
            ])

            # 组装数据
            apps_risk_num, apps_danger_num = 0, 0
            for app_query_i in app_query_mongo:
                if 'apps_risk' in list(app_query_i.keys()) and app_query_i['apps_risk']:
                    apps_risk_num += len(app_query_i['apps_risk'])
                if 'apps_danger' in list(app_query_i.keys()) and app_query_i['apps_danger']:
                    apps_danger_num += len(app_query_i['apps_danger'])

            data['risk'] = apps_risk_num
            data['danger'] = apps_danger_num
            data['total'] = apps_risk_num + apps_danger_num

    return Response(data)